{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import KFold\n",
    "import datetime\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use the leaking data plus as a test\n",
    "df_train = pd.read_csv('../../Large_output/train_leaking_plus.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def features_engineering(df):\n",
    "    \n",
    "    # Sort by localtime\n",
    "    df.sort_values(\"local_time\")\n",
    "    df.reset_index(drop=True)\n",
    "    \n",
    "    # Add more features\n",
    "    df[\"local_time\"] = pd.to_datetime(df[\"local_time\"],format=\"%Y-%m-%d %H:%M:%S\")\n",
    "    df[\"hour\"] = df[\"local_time\"].dt.hour\n",
    "    df[\"weekend\"] = df[\"local_time\"].dt.weekday\n",
    "    df['square_feet'] =  np.log1p(df['square_feet'])\n",
    "    \n",
    "    \n",
    "    # Encode Categorical Data\n",
    "    le = LabelEncoder()\n",
    "    df[\"primary_use\"] = le.fit_transform(df[\"primary_use\"])\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas.api.types import is_datetime64_any_dtype as is_datetime\n",
    "from pandas.api.types import is_categorical_dtype\n",
    "\n",
    "def reduce_mem_usage(df, use_float16=False):\n",
    "    \"\"\"\n",
    "    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    \n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n",
    "            continue\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == \"int\":\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype(\"category\")\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n",
    "    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 3736.60 MB\n",
      "Memory usage after optimization is: 1313.65 MB\n",
      "Decreased by 64.8%\n"
     ]
    }
   ],
   "source": [
    "df_train = reduce_mem_usage(df_train)\n",
    "train_engineer = features_engineering(df_train)\n",
    "train_engineer.loc[(train_engineer['site_id']==0) & (train_engineer['meter']==0),'meter_reading']\\\n",
    "=train_engineer.loc[(train_engineer['site_id']==0) & (train_engineer['meter']==0),'meter_reading'].mul(0.2931)\n",
    "target = np.log1p(train_engineer[\"meter_reading\"])\n",
    "features = train_engineer[['building_id', 'meter','site_id','primary_use', \n",
    "                          'square_feet','air_temperature','cloud_coverage',\n",
    "                          'dew_temperature','precip_depth_1_hr','hour', 'weekend','is_holiday']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from bayes_opt import BayesianOptimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_features = [\"building_id\", \"site_id\", \"meter\", \"primary_use\",  \"weekend\",'is_holiday']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | colsam... |  max_bin  | max_depth | min_ch... | min_ch... | min_sp... | num_le... | reg_alpha | reg_la... | subsample | subsam... |\n",
      "-------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "[25]\tcv_agg's rmse: 1.09657 + 0.000226948\n",
      "[50]\tcv_agg's rmse: 0.799107 + 0.000235839\n",
      "[75]\tcv_agg's rmse: 0.715756 + 0.000252245\n",
      "[100]\tcv_agg's rmse: 0.67395 + 0.000159532\n",
      "[125]\tcv_agg's rmse: 0.658381 + 0.000157569\n",
      "[150]\tcv_agg's rmse: 0.648811 + 0.00014468\n",
      "[175]\tcv_agg's rmse: 0.641947 + 9.12801e-05\n",
      "[200]\tcv_agg's rmse: 0.635277 + 8.97073e-05\n",
      "[225]\tcv_agg's rmse: 0.628499 + 6.78464e-05\n",
      "[250]\tcv_agg's rmse: 0.624807 + 2.29541e-05\n",
      "[275]\tcv_agg's rmse: 0.621711 + 8.18962e-05\n",
      "[300]\tcv_agg's rmse: 0.618956 + 0.000366487\n",
      "[325]\tcv_agg's rmse: 0.617304 + 0.000338922\n",
      "[350]\tcv_agg's rmse: 0.616136 + 0.000241523\n",
      "[375]\tcv_agg's rmse: 0.615207 + 0.000189091\n",
      "[400]\tcv_agg's rmse: 0.614529 + 0.000128495\n",
      "[425]\tcv_agg's rmse: 0.614306 + 0.00022897\n",
      "[450]\tcv_agg's rmse: 0.613937 + 0.000106766\n",
      "[475]\tcv_agg's rmse: 0.613743 + 5.64789e-05\n",
      "[500]\tcv_agg's rmse: 0.61357 + 7.86899e-05\n",
      "[525]\tcv_agg's rmse: 0.613527 + 8.24802e-05\n",
      "[550]\tcv_agg's rmse: 0.61348 + 5.4153e-05\n",
      "[575]\tcv_agg's rmse: 0.613429 + 6.00835e-05\n",
      "[600]\tcv_agg's rmse: 0.613413 + 5.22847e-05\n",
      "[625]\tcv_agg's rmse: 0.613356 + 2.64472e-05\n",
      "[650]\tcv_agg's rmse: 0.613349 + 2.87244e-05\n",
      "[675]\tcv_agg's rmse: 0.613345 + 3.29425e-05\n",
      "[700]\tcv_agg's rmse: 0.613286 + 3.21204e-05\n",
      "[725]\tcv_agg's rmse: 0.613172 + 0.000121877\n",
      "[750]\tcv_agg's rmse: 0.613147 + 0.000151581\n",
      "[775]\tcv_agg's rmse: 0.613136 + 0.000166346\n",
      "[800]\tcv_agg's rmse: 0.613067 + 0.000124572\n",
      "[825]\tcv_agg's rmse: 0.613032 + 0.000101333\n",
      "[850]\tcv_agg's rmse: 0.612976 + 0.000105206\n",
      "[875]\tcv_agg's rmse: 0.61296 + 9.51462e-05\n",
      "[900]\tcv_agg's rmse: 0.612951 + 0.000101455\n",
      "[925]\tcv_agg's rmse: 0.61294 + 0.000115811\n",
      "[950]\tcv_agg's rmse: 0.61293 + 0.000117279\n",
      "[975]\tcv_agg's rmse: 0.612923 + 0.000116395\n",
      "[1000]\tcv_agg's rmse: 0.612918 + 0.000112831\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-0.6129  \u001b[0m | \u001b[0m 0.4851  \u001b[0m | \u001b[0m 548.7   \u001b[0m | \u001b[0m-0.8816  \u001b[0m | \u001b[0m 35.17   \u001b[0m | \u001b[0m 22.53   \u001b[0m | \u001b[0m 0.6978  \u001b[0m | \u001b[0m 2.173e+0\u001b[0m | \u001b[0m 0.2705  \u001b[0m | \u001b[0m 0.3104  \u001b[0m | \u001b[0m 0.8755  \u001b[0m | \u001b[0m 19.13   \u001b[0m |\n",
      "[25]\tcv_agg's rmse: 1.42249 + 0.00342626\n",
      "[50]\tcv_agg's rmse: 1.20876 + 0.00552696\n",
      "[75]\tcv_agg's rmse: 1.09557 + 0.00575699\n",
      "[100]\tcv_agg's rmse: 1.03017 + 0.00566768\n",
      "[125]\tcv_agg's rmse: 0.984794 + 0.000572469\n",
      "[150]\tcv_agg's rmse: 0.956199 + 0.00163088\n",
      "[175]\tcv_agg's rmse: 0.937047 + 0.0030795\n",
      "[200]\tcv_agg's rmse: 0.916814 + 0.00300448\n",
      "[225]\tcv_agg's rmse: 0.900829 + 0.00272359\n",
      "[250]\tcv_agg's rmse: 0.888866 + 0.00382197\n",
      "[275]\tcv_agg's rmse: 0.871249 + 0.00408429\n",
      "[300]\tcv_agg's rmse: 0.861959 + 0.00441866\n",
      "[325]\tcv_agg's rmse: 0.854771 + 0.00459499\n",
      "[350]\tcv_agg's rmse: 0.843464 + 0.00523699\n",
      "[375]\tcv_agg's rmse: 0.833999 + 0.0025891\n",
      "[400]\tcv_agg's rmse: 0.827804 + 0.00244912\n",
      "[425]\tcv_agg's rmse: 0.823035 + 0.00169052\n",
      "[450]\tcv_agg's rmse: 0.815781 + 0.00259963\n",
      "[475]\tcv_agg's rmse: 0.809752 + 0.00197914\n",
      "[500]\tcv_agg's rmse: 0.803509 + 0.00207022\n",
      "[525]\tcv_agg's rmse: 0.79953 + 0.00207229\n",
      "[550]\tcv_agg's rmse: 0.794261 + 0.00060155\n",
      "[575]\tcv_agg's rmse: 0.790873 + 0.00068768\n",
      "[600]\tcv_agg's rmse: 0.787786 + 0.000565809\n",
      "[625]\tcv_agg's rmse: 0.784292 + 0.000432565\n",
      "[650]\tcv_agg's rmse: 0.780513 + 0.000459157\n",
      "[675]\tcv_agg's rmse: 0.777923 + 0.000574627\n",
      "[700]\tcv_agg's rmse: 0.775142 + 0.000810092\n",
      "[725]\tcv_agg's rmse: 0.7727 + 0.00102014\n",
      "[750]\tcv_agg's rmse: 0.769513 + 0.000580806\n",
      "[775]\tcv_agg's rmse: 0.766529 + 0.000521943\n",
      "[800]\tcv_agg's rmse: 0.764475 + 0.000453395\n",
      "[825]\tcv_agg's rmse: 0.762501 + 0.000619091\n",
      "[850]\tcv_agg's rmse: 0.76045 + 0.00087107\n",
      "[875]\tcv_agg's rmse: 0.758408 + 0.000869775\n",
      "[900]\tcv_agg's rmse: 0.756596 + 0.000543876\n",
      "[925]\tcv_agg's rmse: 0.755009 + 0.000540903\n",
      "[950]\tcv_agg's rmse: 0.753261 + 0.00075537\n",
      "[975]\tcv_agg's rmse: 0.751267 + 0.000748014\n",
      "[1000]\tcv_agg's rmse: 0.749378 + 0.00105055\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m-0.7494  \u001b[0m | \u001b[0m 0.715   \u001b[0m | \u001b[0m 359.7   \u001b[0m | \u001b[0m 5.061   \u001b[0m | \u001b[0m 160.0   \u001b[0m | \u001b[0m 22.04   \u001b[0m | \u001b[0m 0.2041  \u001b[0m | \u001b[0m 1.047e+0\u001b[0m | \u001b[0m 0.1924  \u001b[0m | \u001b[0m 1.679   \u001b[0m | \u001b[0m 0.3096  \u001b[0m | \u001b[0m 17.01   \u001b[0m |\n",
      "[25]\tcv_agg's rmse: 1.56249 + 0.000503432\n",
      "[50]\tcv_agg's rmse: 1.38403 + 0.00225654\n",
      "[75]\tcv_agg's rmse: 1.30759 + 0.00632531\n",
      "[100]\tcv_agg's rmse: 1.25697 + 0.00416405\n",
      "[125]\tcv_agg's rmse: 1.22366 + 0.00555607\n",
      "[150]\tcv_agg's rmse: 1.19671 + 0.00524473\n",
      "[175]\tcv_agg's rmse: 1.17522 + 0.00469979\n",
      "[200]\tcv_agg's rmse: 1.1462 + 0.00810544\n",
      "[225]\tcv_agg's rmse: 1.13068 + 0.00814597\n",
      "[250]\tcv_agg's rmse: 1.11406 + 0.010392\n",
      "[275]\tcv_agg's rmse: 1.09615 + 0.00774179\n",
      "[300]\tcv_agg's rmse: 1.08108 + 0.00595757\n",
      "[325]\tcv_agg's rmse: 1.0687 + 0.00697453\n",
      "[350]\tcv_agg's rmse: 1.05734 + 0.00727925\n",
      "[375]\tcv_agg's rmse: 1.04793 + 0.00663224\n",
      "[400]\tcv_agg's rmse: 1.03855 + 0.00318501\n",
      "[425]\tcv_agg's rmse: 1.03108 + 0.00475335\n",
      "[450]\tcv_agg's rmse: 1.02334 + 0.00302739\n",
      "[475]\tcv_agg's rmse: 1.01227 + 0.00269478\n",
      "[500]\tcv_agg's rmse: 1.00645 + 0.00166178\n",
      "[525]\tcv_agg's rmse: 0.999538 + 0.00273572\n",
      "[550]\tcv_agg's rmse: 0.992643 + 0.00518653\n",
      "[575]\tcv_agg's rmse: 0.986574 + 0.0061752\n",
      "[600]\tcv_agg's rmse: 0.978643 + 0.00515231\n",
      "[625]\tcv_agg's rmse: 0.969072 + 0.00884673\n",
      "[650]\tcv_agg's rmse: 0.960994 + 0.00782137\n",
      "[675]\tcv_agg's rmse: 0.956651 + 0.00583025\n",
      "[700]\tcv_agg's rmse: 0.954918 + 0.00570288\n",
      "[725]\tcv_agg's rmse: 0.951576 + 0.00548599\n",
      "[750]\tcv_agg's rmse: 0.947444 + 0.00476428\n",
      "[775]\tcv_agg's rmse: 0.943731 + 0.00462391\n",
      "[800]\tcv_agg's rmse: 0.939132 + 0.00557005\n",
      "[825]\tcv_agg's rmse: 0.935633 + 0.00566436\n",
      "[850]\tcv_agg's rmse: 0.93213 + 0.00472179\n",
      "[875]\tcv_agg's rmse: 0.929162 + 0.00417499\n",
      "[900]\tcv_agg's rmse: 0.926377 + 0.00269253\n",
      "[925]\tcv_agg's rmse: 0.924028 + 0.00344401\n",
      "[950]\tcv_agg's rmse: 0.919963 + 0.00426752\n",
      "[975]\tcv_agg's rmse: 0.91845 + 0.00444872\n",
      "[1000]\tcv_agg's rmse: 0.917242 + 0.00429313\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m-0.9172  \u001b[0m | \u001b[0m 0.5675  \u001b[0m | \u001b[0m 499.2   \u001b[0m | \u001b[0m 3.063   \u001b[0m | \u001b[0m 43.65   \u001b[0m | \u001b[0m 4.28    \u001b[0m | \u001b[0m 0.3285  \u001b[0m | \u001b[0m 1.07e+03\u001b[0m | \u001b[0m 1.216   \u001b[0m | \u001b[0m 0.4357  \u001b[0m | \u001b[0m 0.6169  \u001b[0m | \u001b[0m 7.481   \u001b[0m |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/engine.py:502: UserWarning: Found `num_boost_round` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\tcv_agg's rmse: 1.08394 + 0.0034858\n",
      "[50]\tcv_agg's rmse: 0.83325 + 0.0034035\n",
      "[75]\tcv_agg's rmse: 0.738826 + 0.00203062\n",
      "[100]\tcv_agg's rmse: 0.702647 + 0.00097106\n",
      "[125]\tcv_agg's rmse: 0.688808 + 0.000637009\n",
      "[150]\tcv_agg's rmse: 0.680707 + 0.000601476\n",
      "[175]\tcv_agg's rmse: 0.676684 + 0.000748957\n",
      "[200]\tcv_agg's rmse: 0.672182 + 0.000649037\n",
      "[225]\tcv_agg's rmse: 0.668656 + 0.000401819\n",
      "[250]\tcv_agg's rmse: 0.665841 + 0.000461809\n",
      "[275]\tcv_agg's rmse: 0.663172 + 0.00032243\n",
      "[300]\tcv_agg's rmse: 0.66105 + 0.000322913\n",
      "[325]\tcv_agg's rmse: 0.659422 + 0.000259199\n",
      "[350]\tcv_agg's rmse: 0.65758 + 0.000183106\n",
      "[375]\tcv_agg's rmse: 0.655793 + 0.000147847\n",
      "[400]\tcv_agg's rmse: 0.654786 + 0.000162355\n",
      "[425]\tcv_agg's rmse: 0.653381 + 0.000162061\n",
      "[450]\tcv_agg's rmse: 0.651992 + 0.000151641\n",
      "[475]\tcv_agg's rmse: 0.650722 + 0.00010466\n",
      "[500]\tcv_agg's rmse: 0.649562 + 0.00021921\n",
      "[525]\tcv_agg's rmse: 0.648343 + 0.00036458\n",
      "[550]\tcv_agg's rmse: 0.647389 + 0.000293007\n",
      "[575]\tcv_agg's rmse: 0.646284 + 0.000175302\n",
      "[600]\tcv_agg's rmse: 0.645136 + 0.000147203\n",
      "[625]\tcv_agg's rmse: 0.644243 + 0.000148464\n",
      "[650]\tcv_agg's rmse: 0.64347 + 0.000164502\n",
      "[675]\tcv_agg's rmse: 0.642602 + 0.000157292\n",
      "[700]\tcv_agg's rmse: 0.641635 + 0.00020909\n",
      "[725]\tcv_agg's rmse: 0.640756 + 0.00023594\n",
      "[750]\tcv_agg's rmse: 0.640249 + 0.000253515\n",
      "[775]\tcv_agg's rmse: 0.638977 + 0.000132381\n",
      "[800]\tcv_agg's rmse: 0.638388 + 0.00013728\n",
      "[825]\tcv_agg's rmse: 0.637753 + 0.000155516\n",
      "[850]\tcv_agg's rmse: 0.637216 + 0.000143761\n",
      "[875]\tcv_agg's rmse: 0.636611 + 9.95965e-05\n",
      "[900]\tcv_agg's rmse: 0.635839 + 0.000134993\n",
      "[925]\tcv_agg's rmse: 0.635294 + 0.000139294\n",
      "[950]\tcv_agg's rmse: 0.634784 + 0.000160349\n",
      "[975]\tcv_agg's rmse: 0.634287 + 0.000185884\n",
      "[1000]\tcv_agg's rmse: 0.633798 + 0.000144597\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m-0.6338  \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 180.0   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 200.0   \u001b[0m | \u001b[0m 30.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 3e+03   \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/engine.py:502: UserWarning: Found `num_boost_round` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\tcv_agg's rmse: 1.60752 + 0.00168434\n",
      "[50]\tcv_agg's rmse: 1.45713 + 0.00136349\n",
      "[75]\tcv_agg's rmse: 1.33872 + 0.00154226\n",
      "[100]\tcv_agg's rmse: 1.25597 + 0.000998296\n",
      "[125]\tcv_agg's rmse: 1.21197 + 0.00125744\n",
      "[150]\tcv_agg's rmse: 1.16351 + 0.00246804\n",
      "[175]\tcv_agg's rmse: 1.12552 + 0.00215358\n",
      "[200]\tcv_agg's rmse: 1.10745 + 0.00226326\n",
      "[225]\tcv_agg's rmse: 1.09214 + 0.00106603\n",
      "[250]\tcv_agg's rmse: 1.07428 + 0.000725348\n",
      "[275]\tcv_agg's rmse: 1.0635 + 0.00185776\n",
      "[300]\tcv_agg's rmse: 1.04659 + 0.000484628\n",
      "[325]\tcv_agg's rmse: 1.03776 + 0.00217182\n",
      "[350]\tcv_agg's rmse: 1.03155 + 0.00215185\n",
      "[375]\tcv_agg's rmse: 1.01826 + 0.00203752\n",
      "[400]\tcv_agg's rmse: 1.01068 + 0.00236527\n",
      "[425]\tcv_agg's rmse: 1.00241 + 0.00315624\n",
      "[450]\tcv_agg's rmse: 1.00052 + 0.00312533\n",
      "[475]\tcv_agg's rmse: 0.99436 + 0.00305775\n",
      "[500]\tcv_agg's rmse: 0.988793 + 0.00230185\n",
      "[525]\tcv_agg's rmse: 0.979413 + 0.00413985\n",
      "[550]\tcv_agg's rmse: 0.975127 + 0.003704\n",
      "[575]\tcv_agg's rmse: 0.968732 + 0.00335111\n",
      "[600]\tcv_agg's rmse: 0.964258 + 0.0024473\n",
      "[625]\tcv_agg's rmse: 0.961268 + 0.00263878\n",
      "[650]\tcv_agg's rmse: 0.959027 + 0.00311162\n",
      "[675]\tcv_agg's rmse: 0.957296 + 0.00292556\n",
      "[700]\tcv_agg's rmse: 0.9522 + 0.00168095\n",
      "[725]\tcv_agg's rmse: 0.950473 + 0.00161982\n",
      "[750]\tcv_agg's rmse: 0.947541 + 0.0020815\n",
      "[775]\tcv_agg's rmse: 0.94516 + 0.00221052\n",
      "[800]\tcv_agg's rmse: 0.941125 + 0.00205695\n",
      "[825]\tcv_agg's rmse: 0.938155 + 0.00274997\n",
      "[850]\tcv_agg's rmse: 0.936343 + 0.00272703\n",
      "[875]\tcv_agg's rmse: 0.934062 + 0.00272468\n",
      "[900]\tcv_agg's rmse: 0.931714 + 0.00223218\n",
      "[925]\tcv_agg's rmse: 0.929634 + 0.00256656\n",
      "[950]\tcv_agg's rmse: 0.927919 + 0.00236071\n",
      "[975]\tcv_agg's rmse: 0.925799 + 0.00219917\n",
      "[1000]\tcv_agg's rmse: 0.924634 + 0.00229327\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-0.9246  \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 180.0   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 200.0   \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 1.922e+0\u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/engine.py:502: UserWarning: Found `num_boost_round` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\tcv_agg's rmse: 1.52009 + 0.000272445\n",
      "[50]\tcv_agg's rmse: 1.34825 + 0.000243431\n",
      "[75]\tcv_agg's rmse: 1.21728 + 0.00047436\n",
      "[100]\tcv_agg's rmse: 1.11445 + 0.000822596\n",
      "[125]\tcv_agg's rmse: 1.07251 + 0.000905082\n",
      "[150]\tcv_agg's rmse: 1.02855 + 0.00115529\n",
      "[175]\tcv_agg's rmse: 0.997978 + 0.00107508\n",
      "[200]\tcv_agg's rmse: 0.982865 + 0.00101366\n",
      "[225]\tcv_agg's rmse: 0.969778 + 0.00101394\n",
      "[250]\tcv_agg's rmse: 0.957384 + 0.000939188\n",
      "[275]\tcv_agg's rmse: 0.947669 + 0.000869286\n",
      "[300]\tcv_agg's rmse: 0.937349 + 0.000854704\n",
      "[325]\tcv_agg's rmse: 0.931469 + 0.000910284\n",
      "[350]\tcv_agg's rmse: 0.927577 + 0.000920866\n",
      "[375]\tcv_agg's rmse: 0.920913 + 0.000898511\n",
      "[400]\tcv_agg's rmse: 0.915141 + 0.00111698\n",
      "[425]\tcv_agg's rmse: 0.911031 + 0.000844258\n",
      "[450]\tcv_agg's rmse: 0.909756 + 0.000836171\n",
      "[475]\tcv_agg's rmse: 0.906968 + 0.000820373\n",
      "[500]\tcv_agg's rmse: 0.904705 + 0.000796658\n",
      "[525]\tcv_agg's rmse: 0.901127 + 0.000630176\n",
      "[550]\tcv_agg's rmse: 0.899626 + 0.000705784\n",
      "[575]\tcv_agg's rmse: 0.89785 + 0.000876186\n",
      "[600]\tcv_agg's rmse: 0.896395 + 0.000738302\n",
      "[625]\tcv_agg's rmse: 0.894843 + 0.000614247\n",
      "[650]\tcv_agg's rmse: 0.89433 + 0.000584136\n",
      "[675]\tcv_agg's rmse: 0.893469 + 0.00062417\n",
      "[700]\tcv_agg's rmse: 0.892134 + 0.000732625\n",
      "[725]\tcv_agg's rmse: 0.891617 + 0.000720355\n",
      "[750]\tcv_agg's rmse: 0.890703 + 0.00069689\n",
      "[775]\tcv_agg's rmse: 0.890056 + 0.000641772\n",
      "[800]\tcv_agg's rmse: 0.889121 + 0.000556945\n",
      "[825]\tcv_agg's rmse: 0.8885 + 0.000524888\n",
      "[850]\tcv_agg's rmse: 0.887989 + 0.000498067\n",
      "[875]\tcv_agg's rmse: 0.887402 + 0.000452814\n",
      "[900]\tcv_agg's rmse: 0.886817 + 0.00043755\n",
      "[925]\tcv_agg's rmse: 0.886306 + 0.000428435\n",
      "[950]\tcv_agg's rmse: 0.88584 + 0.000424992\n",
      "[975]\tcv_agg's rmse: 0.885384 + 0.000414184\n",
      "[1000]\tcv_agg's rmse: 0.885151 + 0.000401739\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-0.8852  \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 600.0   \u001b[0m | \u001b[0m-1.0     \u001b[0m | \u001b[0m 20.0    \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 3e+03   \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 20.0    \u001b[0m |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/engine.py:502: UserWarning: Found `num_boost_round` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\tcv_agg's rmse: 0.875095 + 0.000378841\n",
      "[50]\tcv_agg's rmse: 0.693115 + 0.000646731\n",
      "[75]\tcv_agg's rmse: 0.656049 + 0.000549452\n",
      "[100]\tcv_agg's rmse: 0.639551 + 0.000228452\n",
      "[125]\tcv_agg's rmse: 0.628167 + 0.000242477\n",
      "[150]\tcv_agg's rmse: 0.619985 + 4.12991e-05\n",
      "[175]\tcv_agg's rmse: 0.614858 + 0.000311238\n",
      "[200]\tcv_agg's rmse: 0.611383 + 0.000256834\n",
      "[225]\tcv_agg's rmse: 0.608921 + 0.000271018\n",
      "[250]\tcv_agg's rmse: 0.606994 + 0.000254185\n",
      "[275]\tcv_agg's rmse: 0.604972 + 0.000413716\n",
      "[300]\tcv_agg's rmse: 0.603425 + 0.000501505\n",
      "[325]\tcv_agg's rmse: 0.60209 + 0.00047211\n",
      "[350]\tcv_agg's rmse: 0.600838 + 0.000439086\n",
      "[375]\tcv_agg's rmse: 0.599596 + 0.00046043\n",
      "[400]\tcv_agg's rmse: 0.598687 + 0.000509458\n",
      "[425]\tcv_agg's rmse: 0.597334 + 0.000681378\n",
      "[450]\tcv_agg's rmse: 0.596339 + 0.000835793\n",
      "[475]\tcv_agg's rmse: 0.59584 + 0.000970967\n",
      "[500]\tcv_agg's rmse: 0.59574 + 0.00104871\n",
      "[525]\tcv_agg's rmse: 0.59557 + 0.00108031\n",
      "[550]\tcv_agg's rmse: 0.595382 + 0.00104491\n",
      "[575]\tcv_agg's rmse: 0.595321 + 0.000996751\n",
      "[600]\tcv_agg's rmse: 0.595267 + 0.00102702\n",
      "[625]\tcv_agg's rmse: 0.595252 + 0.00101114\n",
      "[650]\tcv_agg's rmse: 0.595198 + 0.0010527\n",
      "[675]\tcv_agg's rmse: 0.595152 + 0.0010754\n",
      "[700]\tcv_agg's rmse: 0.595003 + 0.00123365\n",
      "[725]\tcv_agg's rmse: 0.594957 + 0.00128763\n",
      "[750]\tcv_agg's rmse: 0.59494 + 0.00129347\n",
      "[775]\tcv_agg's rmse: 0.594891 + 0.00134639\n",
      "[800]\tcv_agg's rmse: 0.594888 + 0.00134385\n",
      "[825]\tcv_agg's rmse: 0.594819 + 0.00129685\n",
      "[850]\tcv_agg's rmse: 0.594796 + 0.00130207\n",
      "[875]\tcv_agg's rmse: 0.594759 + 0.00128556\n",
      "[900]\tcv_agg's rmse: 0.594727 + 0.00131421\n",
      "[925]\tcv_agg's rmse: 0.594683 + 0.00130433\n",
      "[950]\tcv_agg's rmse: 0.594675 + 0.00131107\n",
      "[975]\tcv_agg's rmse: 0.594659 + 0.00131951\n",
      "[1000]\tcv_agg's rmse: 0.594638 + 0.00130101\n",
      "| \u001b[95m 7       \u001b[0m | \u001b[95m-0.5946  \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 180.0   \u001b[0m | \u001b[95m-1.0     \u001b[0m | \u001b[95m 20.0    \u001b[0m | \u001b[95m 30.0    \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 2.551e+0\u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 20.0    \u001b[0m |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/engine.py:502: UserWarning: Found `num_boost_round` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\tcv_agg's rmse: 1.08185 + 0.000594016\n",
      "[50]\tcv_agg's rmse: 0.823172 + 0.00288788\n",
      "[75]\tcv_agg's rmse: 0.72673 + 0.00279074\n",
      "[100]\tcv_agg's rmse: 0.687816 + 0.00106829\n",
      "[125]\tcv_agg's rmse: 0.670922 + 0.000891925\n",
      "[150]\tcv_agg's rmse: 0.661754 + 0.000681221\n",
      "[175]\tcv_agg's rmse: 0.655498 + 0.000653556\n",
      "[200]\tcv_agg's rmse: 0.649932 + 0.000390164\n",
      "[225]\tcv_agg's rmse: 0.645186 + 0.000475177\n",
      "[250]\tcv_agg's rmse: 0.641501 + 0.000570042\n",
      "[275]\tcv_agg's rmse: 0.638086 + 0.000476834\n",
      "[300]\tcv_agg's rmse: 0.635326 + 0.000751794\n",
      "[325]\tcv_agg's rmse: 0.632965 + 0.000699091\n",
      "[350]\tcv_agg's rmse: 0.630459 + 0.000470387\n",
      "[375]\tcv_agg's rmse: 0.628387 + 0.00041941\n",
      "[400]\tcv_agg's rmse: 0.626864 + 0.000447909\n",
      "[425]\tcv_agg's rmse: 0.625048 + 0.000488878\n",
      "[450]\tcv_agg's rmse: 0.623421 + 0.000456777\n",
      "[475]\tcv_agg's rmse: 0.621812 + 0.000573358\n",
      "[500]\tcv_agg's rmse: 0.620416 + 0.000624399\n",
      "[525]\tcv_agg's rmse: 0.618967 + 0.000449477\n",
      "[550]\tcv_agg's rmse: 0.617848 + 0.000456179\n",
      "[575]\tcv_agg's rmse: 0.616359 + 0.000428501\n",
      "[600]\tcv_agg's rmse: 0.614631 + 0.000413069\n",
      "[625]\tcv_agg's rmse: 0.613521 + 0.000473253\n",
      "[650]\tcv_agg's rmse: 0.612491 + 0.000432043\n",
      "[675]\tcv_agg's rmse: 0.611367 + 0.00040531\n",
      "[700]\tcv_agg's rmse: 0.610283 + 0.000371997\n",
      "[725]\tcv_agg's rmse: 0.609407 + 0.00045653\n",
      "[750]\tcv_agg's rmse: 0.60868 + 0.000444981\n",
      "[775]\tcv_agg's rmse: 0.607454 + 0.000297791\n",
      "[800]\tcv_agg's rmse: 0.606707 + 0.000289681\n",
      "[825]\tcv_agg's rmse: 0.605938 + 0.000332027\n",
      "[850]\tcv_agg's rmse: 0.60524 + 0.000342897\n",
      "[875]\tcv_agg's rmse: 0.604456 + 0.000335675\n",
      "[900]\tcv_agg's rmse: 0.60346 + 0.000454089\n",
      "[925]\tcv_agg's rmse: 0.602899 + 0.000411676\n",
      "[950]\tcv_agg's rmse: 0.602303 + 0.000443476\n",
      "[975]\tcv_agg's rmse: 0.601685 + 0.000436441\n",
      "[1000]\tcv_agg's rmse: 0.601064 + 0.000515908\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m-0.6011  \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 600.0   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 200.0   \u001b[0m | \u001b[0m 30.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 2.51e+03\u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/engine.py:502: UserWarning: Found `num_boost_round` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\tcv_agg's rmse: 0.923034 + 0.000308308\n",
      "[50]\tcv_agg's rmse: 0.734256 + 0.000498757\n",
      "[75]\tcv_agg's rmse: 0.686558 + 0.000531407\n",
      "[100]\tcv_agg's rmse: 0.665501 + 0.000456955\n",
      "[125]\tcv_agg's rmse: 0.652047 + 0.000322114\n",
      "[150]\tcv_agg's rmse: 0.643082 + 0.000344206\n",
      "[175]\tcv_agg's rmse: 0.637773 + 0.000367545\n",
      "[200]\tcv_agg's rmse: 0.633944 + 0.000502954\n",
      "[225]\tcv_agg's rmse: 0.631392 + 0.000304115\n",
      "[250]\tcv_agg's rmse: 0.629058 + 0.00050757\n",
      "[275]\tcv_agg's rmse: 0.626741 + 0.000716503\n",
      "[300]\tcv_agg's rmse: 0.624889 + 0.000713295\n",
      "[325]\tcv_agg's rmse: 0.623369 + 0.000804687\n",
      "[350]\tcv_agg's rmse: 0.621866 + 0.00090982\n",
      "[375]\tcv_agg's rmse: 0.620706 + 0.000818385\n",
      "[400]\tcv_agg's rmse: 0.61991 + 0.000780897\n",
      "[425]\tcv_agg's rmse: 0.61864 + 0.000614207\n",
      "[450]\tcv_agg's rmse: 0.617751 + 0.000406446\n",
      "[475]\tcv_agg's rmse: 0.61682 + 0.000392485\n",
      "[500]\tcv_agg's rmse: 0.615777 + 0.000415704\n",
      "[525]\tcv_agg's rmse: 0.614912 + 0.000439484\n",
      "[550]\tcv_agg's rmse: 0.614144 + 0.000483132\n",
      "[575]\tcv_agg's rmse: 0.613373 + 0.000515645\n",
      "[600]\tcv_agg's rmse: 0.612528 + 0.000333218\n",
      "[625]\tcv_agg's rmse: 0.611634 + 0.000418868\n",
      "[650]\tcv_agg's rmse: 0.610986 + 0.000350409\n",
      "[675]\tcv_agg's rmse: 0.610267 + 0.00029938\n",
      "[700]\tcv_agg's rmse: 0.609637 + 0.000328898\n",
      "[725]\tcv_agg's rmse: 0.608919 + 0.000280007\n",
      "[750]\tcv_agg's rmse: 0.608498 + 0.000278983\n",
      "[775]\tcv_agg's rmse: 0.607418 + 0.000603594\n",
      "[800]\tcv_agg's rmse: 0.606895 + 0.000654285\n",
      "[825]\tcv_agg's rmse: 0.606318 + 0.000751513\n",
      "[850]\tcv_agg's rmse: 0.605861 + 0.000810602\n",
      "[875]\tcv_agg's rmse: 0.6052 + 0.000872021\n",
      "[900]\tcv_agg's rmse: 0.604561 + 0.00099234\n",
      "[925]\tcv_agg's rmse: 0.604108 + 0.00104451\n",
      "[950]\tcv_agg's rmse: 0.603728 + 0.00108844\n",
      "[975]\tcv_agg's rmse: 0.60325 + 0.00104615\n",
      "[1000]\tcv_agg's rmse: 0.60283 + 0.0010628\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m-0.6028  \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 180.0   \u001b[0m | \u001b[0m-1.0     \u001b[0m | \u001b[0m 20.0    \u001b[0m | \u001b[0m 30.0    \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 1e+03   \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 20.0    \u001b[0m |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/engine.py:502: UserWarning: Found `num_boost_round` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/local/home/ningzesun/.local/lib/python3.6/site-packages/lightgbm/basic.py:1243: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\tcv_agg's rmse: 1.52016 + 0.000230021\n",
      "[50]\tcv_agg's rmse: 1.34831 + 0.000243513\n",
      "[75]\tcv_agg's rmse: 1.21737 + 0.000334685\n",
      "[100]\tcv_agg's rmse: 1.1144 + 0.000671473\n",
      "[125]\tcv_agg's rmse: 1.07239 + 0.000791715\n",
      "[150]\tcv_agg's rmse: 1.02837 + 0.000850426\n",
      "[175]\tcv_agg's rmse: 0.997757 + 0.000790776\n",
      "[200]\tcv_agg's rmse: 0.982632 + 0.000719083\n",
      "[225]\tcv_agg's rmse: 0.969471 + 0.000609933\n",
      "[250]\tcv_agg's rmse: 0.957071 + 0.000569716\n",
      "[275]\tcv_agg's rmse: 0.947248 + 0.000574891\n",
      "[300]\tcv_agg's rmse: 0.937023 + 0.000534257\n",
      "[325]\tcv_agg's rmse: 0.931132 + 0.000627355\n",
      "[350]\tcv_agg's rmse: 0.927221 + 0.000637691\n",
      "[375]\tcv_agg's rmse: 0.920652 + 0.000600282\n",
      "[400]\tcv_agg's rmse: 0.914901 + 0.000766052\n",
      "[425]\tcv_agg's rmse: 0.91078 + 0.000522233\n",
      "[450]\tcv_agg's rmse: 0.909515 + 0.000526609\n",
      "[475]\tcv_agg's rmse: 0.906702 + 0.000530056\n",
      "[500]\tcv_agg's rmse: 0.904394 + 0.00055288\n",
      "[525]\tcv_agg's rmse: 0.900856 + 0.000376226\n",
      "[550]\tcv_agg's rmse: 0.899362 + 0.00040695\n",
      "[575]\tcv_agg's rmse: 0.897611 + 0.000568439\n",
      "[600]\tcv_agg's rmse: 0.896205 + 0.000472369\n",
      "[625]\tcv_agg's rmse: 0.894608 + 0.000336779\n",
      "[650]\tcv_agg's rmse: 0.894075 + 0.000390484\n",
      "[675]\tcv_agg's rmse: 0.893246 + 0.000419374\n",
      "[700]\tcv_agg's rmse: 0.891953 + 0.000547208\n",
      "[725]\tcv_agg's rmse: 0.891454 + 0.000533289\n",
      "[750]\tcv_agg's rmse: 0.890586 + 0.000524861\n",
      "[775]\tcv_agg's rmse: 0.889957 + 0.000497773\n",
      "[800]\tcv_agg's rmse: 0.88906 + 0.00041838\n",
      "[825]\tcv_agg's rmse: 0.888462 + 0.000383677\n",
      "[850]\tcv_agg's rmse: 0.887932 + 0.000377803\n",
      "[875]\tcv_agg's rmse: 0.887363 + 0.00036476\n",
      "[900]\tcv_agg's rmse: 0.886767 + 0.000336226\n",
      "[925]\tcv_agg's rmse: 0.886257 + 0.000323027\n",
      "[950]\tcv_agg's rmse: 0.885789 + 0.000297701\n",
      "[975]\tcv_agg's rmse: 0.885318 + 0.000316698\n",
      "[1000]\tcv_agg's rmse: 0.885087 + 0.000304147\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m-0.8851  \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 180.0   \u001b[0m | \u001b[0m-1.0     \u001b[0m | \u001b[0m 200.0   \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 2.608e+0\u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 20.0    \u001b[0m |\n",
      "=============================================================================================================================================================\n"
     ]
    }
   ],
   "source": [
    "# setup the function for bayes opt\n",
    "def bayes_parameter_opt_lgb(X, y, init_round=3, opt_round=7, n_folds=3, random_seed=6, n_estimators=1000, learning_rate=0.05):\n",
    "    # prepare data\n",
    "    train_data = lgb.Dataset(data=X, label=y,categorical_feature=categorical_features,free_raw_data=False)\n",
    "    # parameters\n",
    "    def lgb_eval(num_leaves, colsample_bytree, subsample, max_depth, reg_lambda, reg_alpha, min_split_gain, min_child_weight, \n",
    "                min_child_sample, max_bin, subsample_freq):\n",
    "        params = {'objective':'regression','boosting_type': 'gbdt','nthread': 4, 'verbose': -1,\\\n",
    "                  'num_boost_round': n_estimators, 'learning_rate':learning_rate}\n",
    "        params['subsample_freq']=int(round(subsample_freq))\n",
    "        params['min_child_sample']=int(round(min_child_sample))\n",
    "        params['max_bin']=int(round(max_bin))\n",
    "        params[\"num_leaves\"] = int(round(num_leaves))\n",
    "        params['colsample_bytree'] = max(min(colsample_bytree, 1), 0)\n",
    "        params['subsample'] = max(min(subsample, 1), 0)\n",
    "        params['max_depth'] = int(round(max_depth))\n",
    "        params['reg_lambda'] = max(reg_lambda, 0)\n",
    "        params['reg_alpha'] = max(reg_alpha, 0)\n",
    "        params['min_split_gain'] = min_split_gain\n",
    "        params['min_child_weight'] = min_child_weight\n",
    "        cv_result = lgb.cv(params, train_data, nfold=n_folds, seed=random_seed, stratified=False, verbose_eval=25, metrics=['rmse'],early_stopping_rounds=50)\n",
    "        return -1.0 * np.min(cv_result['rmse-mean'])\n",
    "    # range \n",
    "    lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (1000, 3000),\n",
    "                                            'colsample_bytree': (0.1, 0.9),\n",
    "                                            'subsample': (0.1, 0.9),\n",
    "                                            'max_depth': (-1, 12),\n",
    "                                            'reg_lambda': (0.1, 3),\n",
    "                                            'reg_alpha': (0.1, 3),\n",
    "                                            'min_child_sample':(20,200),\n",
    "                                            'max_bin':(180,600),\n",
    "                                            'subsample_freq':(1,20),\n",
    "                                            'min_split_gain': (0.1, 0.9),\n",
    "                                            'min_child_weight': (3, 30)})\n",
    "    # optimize\n",
    "    lgbBO.maximize(init_points=init_round, n_iter=opt_round)\n",
    "\n",
    "opt_params = bayes_parameter_opt_lgb(features, target, init_round=3, opt_round=7, n_folds=3, random_seed=6, n_estimators=1000, learning_rate=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "|   iter    |  target   | colsam... |  max_bin  | max_depth | min_ch... | min_ch... | min_sp... | num_le... | reg_alpha | reg_la... | subsample | subsam... |\n",
    "|  7        | -0.5946   |  0.9      |  180.0    | -1.0      |  20.0     |  30.0     |  0.9      |  2.551e+0 |  0.1      |  0.1      |  0.9      |  20.0     |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"objective\": \"regression\",\n",
    "    \"boosting\": \"gbdt\",\n",
    "    \"num_leaves\": 2551,\n",
    "    \"learning_rate\": 0.05,\n",
    "    \"colsample_bytree\": 0.9,\n",
    "    \"reg_lambda\": 0.1,\n",
    "    'reg_alpha':0.1,\n",
    "    \"metric\": \"rmse\",\n",
    "    'max_bins':180,\n",
    "    'max_depth':-1,\n",
    "    'min_child_sample':20,\n",
    "    'min_child_weight':30,\n",
    "    'min_split_gain':0.9,\n",
    "    'subsample':0.9,\n",
    "    'subsample_freq':20,\n",
    "     \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
